# -*- coding: utf-8 -*-
#easy_install mechanize
import mechanize
import cookielib
import time
from bs4 import BeautifulSoup,Tag
from urllib import myquote_plus
from langconv import *
import random
from urllib import myquote_plus
# Browser
 
__all__ = ['InitBrowser','brYahoo_search','brYahoo_sleep']

__yahoobr = None

def InitBrowser(proxies= True):
    global __yahoobr
    if __yahoobr:
        __yahoobr.close()
        
    __yahoobr = mechanize.Browser()
    # Cookie Jar
    cj = cookielib.LWPCookieJar()
    __yahoobr.set_cookiejar(cj)
    
    # Browser options
    __yahoobr.set_handle_equiv(True)
    __yahoobr.set_handle_gzip(True)
    __yahoobr.set_handle_redirect(True)
    __yahoobr.set_handle_referer(True)
    __yahoobr.set_handle_robots(False)
    
    # Follows refresh 0 but not hangs on refresh > 0
    __yahoobr.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    # Want debugging messages?
    #__yahoobr.set_debug_http(True)
    #__yahoobr.set_debug_redirects(True)
    #__yahoobr.set_debug_responses(True)
    
    # User-Agent (this is cheating, ok?)
    __yahoobr.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
    if proxies:
        __yahoobr.set_proxies({"http": "127.0.0.1:8087"})
    
#现在已经创建好了__br这个浏览器对象，可以用它来打开页面，或者进行交互。
#

InitBrowser(False)

#if True:
def brYahoo_search(query,NumQue=25):
    url_search = "https://search.yahoo.com/search?p=%(query)s&toggle=1&cop=mss&ei=UTF-8&fr=yfp-t-254"

    #query = u'北京航空航天大学'
    #NumQue = 25
    
    query = myquote_plus(query, encoding='utf-8')
    url = url_search % vars()
    
    global __yahoobr
    r = __yahoobr.open(url)
    #print '@\t',__yahoobr.title()
    html = r.read()
    
    fd = open('c:\\cut\\test_yahoo_info.html','w')
    fd.write(html)
    fd.close()
    
    #--------------------------------------
    soup = BeautifulSoup(html) 
    outs=soup.findAll('ol')  
    
    res_temp = []
    for out in outs:
        if not out.attrs.get('start') is None:
            res_temp = out.contents
    #--------------------------------------
    res = []
    while True:
        for resitm in res_temp:
            title_nd = resitm.find('h3')
            abstr_nd = None
            for ast in resitm.findAll('div'):
                if not ast.attrs.get("class") is None and 'abstr' in  ast.attrs["class"]:
                    abstr_nd = ast
                    break
            tmp =  title_nd.find('a')
            title_text = tmp.text
            title_link = tmp.attrs['href']
            abstr_text = ''
            if abstr_nd:
                abstr_text = abstr_nd.text
            res.append( (title_text,title_link,abstr_text) )
        #--------------------------------------
        if len(res)<NumQue:
            nav = soup.find('div',id="pg")
            links_temp = nav.findAll('a')
            next_page_lnk = None
            if links_temp[-1].text == u'Next':
                next_page_lnk = links_temp[-1].attrs['href']
                time.sleep(0.1*random.randint(0, 100)/100.0)
                r=__yahoobr.open(next_page_lnk)
                html = r.read()
            else:
                break
        else:
            break
    #--------------------------------------
    return  res  
    
    
def brYahoo_sleep(max_time=5.0): 
    sleep_time  = max([max_time*random.randint(0, 100)/100.0,2])
    print '@\t begin Yahoo waiting',sleep_time
    time.sleep(sleep_time)
    print '@\t end Yahoo waiting',sleep_time

